# -*- coding: utf-8 -*-

# Scrapy settings for cnhnb_realtime project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     https://docs.scrapy.org/en/latest/topics/settings.html
#     https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
#     https://docs.scrapy.org/en/latest/topics/spider-middleware.html

BOT_NAME = 'cnhnb_realtime'

SPIDER_MODULES = ['cnhnb_realtime.spiders']
NEWSPIDER_MODULE = 'cnhnb_realtime.spiders'

# 更保守的爬取设置，避免503错误
CONCURRENT_REQUESTS = 1  # 降低并发数
CONCURRENT_REQUESTS_PER_DOMAIN = 1
CONCURRENT_REQUESTS_PER_IP = 1

# 增加延迟，避免触发反爬
DOWNLOAD_DELAY = 10  # 增加到10秒
RANDOMIZE_DOWNLOAD_DELAY = True
DOWNLOAD_DELAY_RANDOMIZE = 5  # 随机延迟5秒

# 自动限速
AUTOTHROTTLE_ENABLED = True
AUTOTHROTTLE_START_DELAY = 15  # 初始延迟15秒
AUTOTHROTTLE_MAX_DELAY = 30  # 最大延迟30秒
AUTOTHROTTLE_TARGET_CONCURRENCY = 0.5  # 目标并发0.5
AUTOTHROTTLE_DEBUG = True

# 重试设置
RETRY_ENABLED = True
RETRY_TIMES = 3  # 减少重试次数
RETRY_HTTP_CODES = [500, 502, 503, 504, 408, 429]

# 请求超时
DOWNLOAD_TIMEOUT = 30
RANDOMIZE_DOWNLOAD_DELAY = True

# 禁用缓存
HTTPCACHE_ENABLED = False

# 启用自定义中间件
DOWNLOADER_MIDDLEWARES = {
    'cnhnb_realtime.middlewares.RandomUserAgentMiddleware': 400,
    'cnhnb_realtime.middlewares.RandomDelayMiddleware': 410,
    'cnhnb_realtime.middlewares.RetryMiddleware': 420,
    'cnhnb_realtime.middlewares.AntiDetectionMiddleware': 430,
    'cnhnb_realtime.middlewares.LoggingMiddleware': 440,
    'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': None,
    'scrapy.downloadermiddlewares.retry.RetryMiddleware': None,
}

# 启用管道
ITEM_PIPELINES = {
   'cnhnb_realtime.pipelines.CnhnbRealtimePipeline': 300,
}

# 日志设置
LOG_LEVEL = 'INFO'
LOG_FILE = 'cnhnb_realtime_spider.log'

# 遵守robots.txt
ROBOTSTXT_OBEY = False

# 启用和配置HTTP压缩
COMPRESSION_ENABLED = True

# 设置请求头
DEFAULT_REQUEST_HEADERS = {
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
    'Accept-Encoding': 'gzip, deflate, br',
    'Connection': 'keep-alive',
    'Upgrade-Insecure-Requests': '1',
    'Sec-Fetch-Dest': 'document',
    'Sec-Fetch-Mode': 'navigate',
    'Sec-Fetch-Site': 'none',
    'Cache-Control': 'max-age=0',
}

# 启用自动限速扩展
EXTENSIONS = {
    'scrapy.extensions.telnet.TelnetConsole': None,
}

# 启用自动限速
AUTOTHROTTLE_ENABLED = True
AUTOTHROTTLE_START_DELAY = 15
AUTOTHROTTLE_MAX_DELAY = 30
AUTOTHROTTLE_TARGET_CONCURRENCY = 0.5
AUTOTHROTTLE_DEBUG = True

# 启用统计信息收集
STATS_CLASS = 'scrapy.statscollectors.MemoryStatsCollector'

# 启用内存使用监控
MEMUSAGE_ENABLED = True
MEMUSAGE_LIMIT_MB = 2048
MEMUSAGE_WARNING_MB = 1024

# 启用自动限速
AUTOTHROTTLE_ENABLED = True
AUTOTHROTTLE_START_DELAY = 15
AUTOTHROTTLE_MAX_DELAY = 30
AUTOTHROTTLE_TARGET_CONCURRENCY = 0.5
AUTOTHROTTLE_DEBUG = True

# 启用自动限速
AUTOTHROTTLE_ENABLED = True
AUTOTHROTTLE_START_DELAY = 15
AUTOTHROTTLE_MAX_DELAY = 30
AUTOTHROTTLE_TARGET_CONCURRENCY = 0.5
AUTOTHROTTLE_DEBUG = True

# 请求头设置
USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'

# 中间件设置 - 使用自定义中间件
DOWNLOADER_MIDDLEWARES = {
   'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': None,
   'cnhnb_realtime.middlewares.RandomUserAgentMiddleware': 400,
   'cnhnb_realtime.middlewares.AntiDetectionMiddleware': 410,
   'cnhnb_realtime.middlewares.RandomDelayMiddleware': 420,
   'cnhnb_realtime.middlewares.RetryMiddleware': 90,
   'cnhnb_realtime.middlewares.CustomHttpErrorMiddleware': 95,
   'cnhnb_realtime.middlewares.LoggingMiddleware': 100,
   'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 110,
   'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 400,
   'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware': 350,
   'scrapy.downloadermiddlewares.redirect.RedirectMiddleware': 600,
   'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 810,
   'scrapy.downloadermiddlewares.cookies.CookiesMiddleware': 700,
}

# Redis 设置
REDIS_HOST = '192.168.88.130'
REDIS_PORT = 6379
REDIS_DB = 0
REDIS_PASSWORD = '123456'

# Kafka 设置
KAFKA_BOOTSTRAP_SERVERS = ['192.168.88.130:9092', '192.168.88.131:9092', '192.168.88.132:9092']
KAFKA_TOPIC = 'cnhnb_price_data'

# Spark 设置
SPARK_MASTER = 'spark://192.168.88.130:7077'
SPARK_APP_NAME = 'cnhnb_realtime_processor'

# 缓存设置
HTTPCACHE_EXPIRATION_SECS = 0
HTTPCACHE_DIR = 'httpcache'
HTTPCACHE_IGNORE_HTTP_CODES = []
HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'

# 深度限制
DEPTH_LIMIT = 2
DEPTH_PRIORITY = 0

# Cookie设置
COOKIES_ENABLED = True
COOKIES_DEBUG = False

# Referer设置
REFERER_ENABLED = True

# 请求指纹设置
REQUEST_FINGERPRINTER_IMPLEMENTATION = '2.7'

# 下载器统计
DOWNLOADER_STATS = True

# 启用telnet控制台
TELNETCONSOLE_ENABLED = True

# 自定义中间件配置
RANDOM_DELAY_MIDDLEWARE_DELAY = 2
RETRY_MIDDLEWARE_MAX_RETRY_TIMES = 8 